{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic 回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6    148.0           72.0           35.0    125.0  33.6   \n",
       "1            1     85.0           66.0           29.0    125.0  26.6   \n",
       "2            8    183.0           64.0           29.0    125.0  23.3   \n",
       "3            1     89.0           66.0           23.0     94.0  28.1   \n",
       "4            0    137.0           40.0           35.0    168.0  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "train = pd.read_csv(\"FE_diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 768 non-null int64\n",
      "Glucose                     768 non-null float64\n",
      "BloodPressure               768 non-null float64\n",
      "SkinThickness               768 non-null float64\n",
      "Insulin                     768 non-null float64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null int64\n",
      "Outcome                     768 non-null int64\n",
      "dtypes: float64(6), int64(3)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  get labels\n",
    "y_train = train['Outcome']   \n",
    "X_train = train.drop([\"Outcome\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.48797856 0.53011593 0.4562292  0.422546   0.48392885]\n",
      "cv logloss is: 0.47615970944434044\n"
     ]
    }
   ],
   "source": [
    "#交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv= 5, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优\n",
    "这部分用交叉验证GridSearchCV、LogisticRegressionCV任意一种方式均可"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "目标函数为：J = C* sum(logloss(f(xi), yi)) +  penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "设置候选参数集合\n",
    "生成一个GridSearchCV的实例\n",
    "调用GridSearchCV的fit函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty, tuned_parameters, cv= 5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4760234770646705\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果最佳值在候选参数的边缘，最好再尝试更大的候选参数或更小的候选参数，直到找到拐点。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8VFX+//HXZya9UBMw9CIoNZQsICg2REQMhCYoUqyrKKC7rG1/FvC7goXFXXFVUEQUEQtFUBBURJAWMHQQREpooYUE0jPn98dMIITApE1uJvk89zEP7tx77sz7gptPbjnniDEGpZRS6kpsVgdQSilV9mmxUEop5ZYWC6WUUm5psVBKKeWWFgullFJuabFQSinllhYLpZRSbmmxUEop5ZYWC6WUUm75WB2gpISFhZkGDRpYHUMppbzKhg0bThhjwt2182ixEJEewFuAHZhmjJmQZ/u/gZtdb4OAGsaYKq5tw4B/ura9YoyZcaXvatCgAbGxsSUZXymlyj0R2V+Qdh4rFiJiB6YAtwHxwHoRWWCM2Z7TxhjzZK72TwBtXcvVgBeBKMAAG1z7nvZUXqWUUpfnyXsWHYA9xpi9xpgMYDbQ+wrtBwOfuZZvB5YaY065CsRSoIcHsyqllLoCTxaL2sDBXO/jXesuISL1gYbAj4XdVymllOd58p6F5LPucuOhDwK+NMZkF2ZfEXkYeBigXr16RcmolCoHMjMziY+PJy0tzeooZVZAQAB16tTB19e3SPt7sljEA3Vzva8DHL5M20HAyDz73pRn3+V5dzLGvA+8DxAVFaUTcyhVQcXHxxMaGkqDBg0Qye93zYrNGMPJkyeJj4+nYcOGRfoMT16GWg80EZGGIuKHsyAsyNtIRK4BqgKrc61eAnQXkaoiUhXo7lqnlFKXSEtLo3r16looLkNEqF69erHOvDx2ZmGMyRKRx3H+kLcDHxpjtonIOCDWGJNTOAYDs02uKfuMMadEZDzOggMwzhhzylNZlVLeTwvFlRX378ej/SyMMd8C3+ZZ90Ke9y9dZt8PgQ89Fs4lOyuL9dNGUa/HaGo1uMbTX6eUKiPufs95MePzR66zOIl3qPDDfRz+cxvNj87D96Pb2b15jdVxlFJeKiQk5Pxyjx49qFKlCr169cq37ciRI2nTpg3NmzcnMDCQNm3a0KZNG7788stCfefGjRtZvHhxsXIXVIUvFnWbRHJq4HwMQs2vYvjtl2+sjqSU8nJjx45l5syZl90+ZcoU4uLi+Pbbb2ncuDFxcXHExcXRv3//Qn2PFotS1qD5X5AHl5Jor07zZSNYteADqyMppbzYrbfeSmhoaJH23b17N7fffjvt27ena9eu/P777wDMnj2bli1bEhkZyc0330xqairjxo3j008/LdJZSWGVm4EEiyu8ztWcHfUTB96J5roNf2NZ4jFuve9ZvWmmlJd5+ZttbD+c5Lbd9iPONjn3Lq6kea1KvHhXi2JnK4iHH36YadOm0bhxY1atWsXjjz/O999/z8svv8zy5cupWbMmiYmJBAYG8sILL7B161YmT57s8VxaLHIJqRJOg6eWsmPKALrtncjSKUe46a+T8fWxWx1NKVUBJCYmsmbNGvr163d+XVZWFgBdunRh6NChDBgwgL59+5Z6Ni0WefgGhNB8zAK2vPcAtyV8zE//Pk7UyOmEBgVaHU0pVQAFPQMoi09DGWMICwsjLi7ukm1Tp05l7dq1LFy4kMjISDZv3lyq2fSeRT7E7kurR2ew7eqHufncd2z9d2+OndRuHkopz6patSoRERHMnTsXAIfDwaZNmwDYu3cvnTp1Yvz48VStWpVDhw4RGhpKcnJyqWTTYnE5IrQY8jq/R71Ex4x1HHu7B3v2H7A6lVLKC9xwww0MGDCAH374gTp16rBkScEHoJg9ezbvvvsukZGRtGjRgoULFwLw5JNP0qpVK1q1akW3bt1o2bIlt9xyC5s2baJt27Yev8EtuTpOe7WoqCjjqcmPDqz8jKuWPc5BapLY93Pat27lke9RShXNjh07aNasWaH2KYuXoTwtv78nEdlgjIlyt6/esyiAetcPJqFSOFd9fR/BX0Xz05np3HzDTVbHUkoVQ0UqEiVBL0MVUI3W3XAM/w4/O7RbNpj587+gvJyVKaWUO1osCiG0QRuCHvuJVL/q9Nj4KJ99/D+yHVowlFLlnxaLQgoIa0CN0cs5EdKUu/c+x6x3XiI1I9v9jkop5cW0WBSBLSSM2qOXcji8C/edmMz8yY9zMlln6FJKlV9aLIrKL5i6j84jvn5fBqXMYtVb97Evwf0QA0qpMmL6nc6XKhAtFsVh96XO8A850voxorO+Z+87/Yjbe8TqVEopC5T2EOVz587l9ddfL3bugtJHZ4tLhIi+r3KicgQ3/fICG2f05sc+M7mlrU6kpFRFNXbsWFJSUnjvvffy3T5lyhQA9u3bR69evfId3gOc40L5+OT/YzomJqZkwhaQnlmUkLBbR3G21/tEyh/UntuXr35aa3UkpZRFijNE+fXXX8/zzz9P165defvtt5k/fz4dO3akbdu2dO/enYSEBACmTZvGmDFjABgyZAijR4+mc+fONGrU6PxwISVJzyxKUKWogaRVCqPeZ/cSsnwwU0+9zQMxd2Cz6TDnSpWa756Bo1vctzvqGoivIPctrmoFd0woXq5CSEpKYsWKFQCcPn2a6OhoRIR3332XN998k4kTJ16yT0JCAqtWrWLLli0MHDiwxM889MyihAU0vQXfBxdTyRcGbH6Qtz76hPQsfbRWKVVwgwYNOr984MABunfvTqtWrZg0aRLbtm3Ld58+ffogIrRu3ZpDhw6VeCY9s/AAn9qRhDz2A2emRvPo/if5zzvHefihJ6gc6Gt1NKXKv4KeAeScUYxY5LksRRQcHHx+eeTIkTz33HP07NmTZcuWMWFC/sfn7+9/ftkTo0vomYWHSLWGVHn8J1KqXsNTJ8fxwX9e4lBiqtWxlFJe5syZM9SuXRtjDDNmzLAshxYLTwoOo9qjS0iqdT1Ppb7NN/99km2HEq1OpZTysOIMUZ7XSy+9RExMDDfeeCM1a9YswZSFo0OUl4bsTM58/giVf/+Kz0x36tzzX2645iqrUylVbhRliPKyfBnKU4ozRLmeWZQGuy+VB03jbPvHGCzfc+7T+/h63R9Wp1KqYhuxqEIViuLSYlFabDZC7nqVtFvG08O2jloL7+XdJRt1mHOllFfQYlHKArqOIqvPVKJsu7lx1VD+9flPZGY7rI6llFJXpMXCAj5tBmIf8gWNfU4ybMdDvPDB15xLz7I6llJKXZYWC4vI1bfg9+C3VPd3MPbQaF6YMp0EHeZcKVVGabGwUq22BP71BwJCq/HKmed4/b//YU/CWatTKVUhjFg8ghGLR1gdw2t4tFiISA8R2SUie0Tkmcu0GSgi20Vkm4jMyrU+W0TiXK8FnsxpqWqNCPrrD5iwprya8Sofv/N/rN93yupUSqlCyhmiPC4ujuuuu44WLVrQunVrPv/880valsQQ5QAbN25k8eLFJZLfHY8N9yEidmAKcBsQD6wXkQXGmO252jQBngW6GGNOi0iNXB+Raoxp46l8ZUpIDYIeXkzqJ/cw7uA7vPnBKRL6v8idkbWsTqaUKqSgoCA+/vhjmjRpwuHDh2nfvj233347VapUOd+moEOUu7Nx40a2bt1Kjx49SiT7lXjyzKIDsMcYs9cYkwHMBnrnafMQMMUYcxrAGJPgwTxlm38ogcO+IqNZX/5mn83xL8YwbcUeq1MppQqpadOmNGnSBIBatWpRo0YNjh8/XuD9d+/eze2330779u3p2rUrv//+OwCzZ8+mZcuWREZGcvPNN5Oamsq4ceP49NNPi3RWUlieHEiwNnAw1/t4oGOeNk0BRGQVYAdeMsbknFMFiEgskAVMMMbM82DWssHHD78BH5C1uCbD1/2PhUvP8MrpCTx7VxvsOsy5UgUycd1Edp7a6bZdTpuC3Le4ttq1PN3h6UJnWbduHRkZGTRu3LjA+zz88MNMmzaNxo0bs2rVKh5//HG+//57Xn75ZZYvX07NmjVJTEwkMDCQF154ga1btzJ58uRCZyssTxaL/H665e2B5gM0AW4C6gC/iEhLY0wiUM8Yc1hEGgE/isgWY8xF3Z5F5GHgYYB69eqVdH5r2Gz49JyAo1Itei37f1SLfYy/J07g1XuuJ8DXbnU6pVQBHTlyhPvuu48ZM2ZgsxXsIk5iYiJr1qyhX79+59dlZTkfq+/SpQtDhw5lwIAB9O3b1yOZr8STxSIeqJvrfR3gcD5t1hhjMoE/RWQXzuKx3hhzGMAYs1dElgNtgYuKhTHmfeB9cI4N5YmDsIrt+lEQWoNO80ZS5Y8nGPnev3hjRHeqBvtZHU2pMq2gZwA5ZxTTe0wv8QxJSUnceeedvPLKK3Tq1KnA+xljCAsLy/cextSpU1m7di0LFy4kMjKSzZs3l2Rktzx5z2I90EREGoqIHzAIyPtU0zzgZgARCcN5WWqviFQVEf9c67sA26loIgdhu+dzmvoe56XjYxg95QsOnkqxOpVS6goyMjKIiYk5fxZQGFWrViUiIuL8tKgOh4NNmzYBsHfvXjp16sT48eOpWrUqhw4dIjQ0lOTk5BI/hvx4rFgYY7KAx4ElwA5gjjFmm4iME5FoV7MlwEkR2Q78BIw1xpwEmgGxIrLJtX5C7qeoKpQm3fC5fyFXBWQzOeVpnp8yg83xOsy5UmXVnDlzWLFiBR999NH5R2IL87TT7Nmzeffdd4mMjKRFixYsXLgQgCeffJJWrVrRqlUrunXrRsuWLbnlllvYtGkTbdu29fgNbh2i3Fuc/IPMGX3ISjrGqOynGHzvCG651rqx7ZUqS4oyRLknL0OVVTpEeUVQvTG+Dy3DJ7wJ/7O/xsKZ/2bW2gNWp1LKa03vMb1CFYri0mLhTUJr4vvAd0j9zkzy/R/7FvyLNxbvvDDM+fQ7L0zoopRSJUiLhbcJqIT9vq9wNI/hOd/PqLLyJf7++W9kZDnYduQM246csTqhUqoc0mLhjXz8sfX/ENPhER70+Y6u257nwemrOGf8rU6mlCqntFh4K5sNuWMi3Poive2/8vDBZ3jxXH9OOEKtTqaUKoe0WHgzEbjhKej9Dl3sO3jD53/861xvElMyrE6mVJm3/76h7L9vqNUxvIYWi/Kg7b3I4NlcY4vnPd9JzFvjflwcpVTJKu0hyufOncvrr79eYvnd8eRwH6o0Ne1OvE89GmT9yek1M3Hc9Bo2HXxQqVJXkkOUZ2Vl4eOT/4/pmJiYkg9/BXpmUY6ck2BOUIXb0r5n5Z4TVsdRqkIq7hDl119/Pc8//zxdu3bl7bffZv78+XTs2JG2bdvSvXt3EhKcMzlMmzaNMWPGADBkyBBGjx5N586dadSo0fnhQkqSnllQjnpyipBmC6Yl+3jl56V0bXqP1YmUKnVH//Uv0ne4vxSbttPZpiD3LfybXctVzz1X6CxFGaIcnAMRrlixAoDTp08THR2NiPDuu+/y5ptvMnHixEv2SUhIYNWqVWzZsoWBAweW+JmHFotypEVEZXAEk3noBI0OfMnhxBhqVQm0OpZSFVJRhijPMWjQoPPLBw4cYODAgRw9epT09HSaNm2a7z59+vRBRGjdujWHDh0qVvb8VPhi4TAOmszfxDXH7OD5mQk9z+ZDRtNo7tqxkOmrdzLqjrZWJ1KqVBX0DCDnjKL+zI9LPENRhyjPERwcfH555MiRPPfcc/Ts2ZNly5YxYcKEfPfx97/Qz8oTY/5V+HsWB47vIdWRzq8Ny8/jpsHXPUCopJIYO4eMLIfVcZSqUIozRHl+zpw5Q+3atTHGMGPGjBJIWDQVvljUzgph4C8Omu7LZNepXVbHKZ4Ri5yvep04V6kxvTK/5/vtR61OpVSFUtwhyvN66aWXiImJ4cYbb6RmTetGmtYhyoFfurQm7GQmM//emv97YDYi3v/IqWPVf7Et/Sd/D3+XN0YOtjqOUh5VlCHKPXkZqqzSIcqL6ZOxkZwNttFuzmZ+3P+D1XFKhK3NPWSLDy2OzGX3sdKZSUspb1J/5scVqlAUlxYL4L0+M2n41LO0OADffTKe9Ox0qyMVX3B1spreSYx9JZ+v3m11GqWUl9Ni4VJ94N1k172KHt8m8Mnmj6yOUyL8O4ygipwj+be5nEvPsjqOUh5VXi6pe0px/360WLiIry/1n32B2qdgz8fvcjyl4D0uy6yGN5IeUpc+jmXMjztsdRqlPCYgIICTJ09qwbgMYwwnT54kICCgyJ9R4ftZ5BZy803Y2remz8+beWflG7zY/dJekl7FZsOvw3Cu+3E8H6z6lcEd6paLm/dK5VWnTh3i4+MLNaxGRRMQEECdOnWKvL8Wi1xEhHrPvoCjf398P/uGre3uo2VYS6tjFYu0HYLjp38RdWohGw/cTvv6Va2OpFSJ8/X1pWHDhlbHKNf0MlQegS1bENSrJ73WG/63ZJz3n9aGXoXj6u4MsK9g9uo9VqdRSnkpLRb5qPXU37DbfIj8eivf/fmd1XGKzecvI6guZ0jduohT58pPT3WlVOnRYpEP31q1CBs2nBu2Gb5YMIGUzBSrIxXP1d3IDI5ggPzAnNiDVqdRSnkhLRaXEfbII5gqlbjz2+N8tNXLhy632fGNGsoN9i0sW70eh8PLL60ppUqdFovLsIeEEDF6DC0OwG9zp3Lk7BGrIxVP2yEIcMPZJfy8W58YUUoVjhaLK6jSvz+2BvUYvCyDyevetDpO8VSph2l0C3f7/Mys1XutTqOU8jJaLK5AfH2p9Y9niDhlyJ73HRuPbbQ6UrHYooZxFSfJ3r2M+NNefh9GKVWqtFi4EXLzTfh3iGLgSpi84l84jBfPD9H0DrKDwhhk+5HP1h2wOo1SyotosXBDRIh4+hlCUxw0W7Sd+XvmWx2p6Hz8sLe9l1vtv7Fs3WbSs7KtTqSU8hIeLRYi0kNEdonIHhF55jJtBorIdhHZJiKzcq0fJiK7Xa9hnszpTmCLFlSKjqZXLMz8cRJnM85aGad42g3DjoNb05ayeKtOjKSUKhiPFQsRsQNTgDuA5sBgEWmep00T4FmgizGmBTDGtb4a8CLQEegAvCgilo5TUePJMdhtPvT4/iRTt0y1MkrxVG+MqX89Q/x+ZtbqfRaHUUp5C0+eWXQA9hhj9hpjMoDZQO88bR4CphhjTgMYYxJc628HlhpjTrm2LQV6eDCrW74REYSNuJ8bthlWLpvBwSTv7dwm7YdTyxzDfnAlO48mWR1HKeUFPFksagO5f6LGu9bl1hRoKiKrRGSNiPQoxL6lrvpDDyJVqzDkhyzeWP+61XGKrtldOAKqcK/PT3y6Rm90K6Xc82SxyG8s7Lxdh32AJsBNwGBgmohUKeC+iMjDIhIrIrGlMTSxPSSEmqNGce2BbJJ+/IE1R9Z4/Ds9wjcAW+QgbrfH8uPG7ZzViZGUUm54sljEA3Vzva8D5J2BJx6Yb4zJNMb8CezCWTwKsi/GmPeNMVHGmKjw8PASDX85VQYMwLdRQ4b/bOP11RPIcnjpD9p2w/AxmfTIXs7c3w5ZnUYpVcZ5slisB5qISEMR8QMGAQvytJkH3AwgImE4L0vtBZYA3UWkquvGdnfXOsuJjw81x46lxoksGi7fzZe/f2l1pKKp2RxT5y8M9/+ZT1fv8/6h2JVSHuWxYmGMyQIex/lDfgcwxxizTUTGiUi0q9kS4KSIbAd+AsYaY04aY04B43EWnPXAONe6MiHkppsI6tiRwatsfLjmv5xJP2N1pCKRdsOo64gnJCGW2P2nrY6jlCrDpLz8RhkVFWViY2NL7ftSt21jX/8BzO8k8Oh9PNMh324kZVvGOcwbTVmQ0Y4frx3HW4PaWp1IKVXKRGSDMSbKXTvtwV1EgS1aUDk6ml7rYdnaz/gj8Q+rIxWeXzDSagA9ZS2/bNnDibPpVidSSpVRWiyKIXzMaOx2H+79GV5f/7p3XvdvPwxfk05PVunESEqpy9JiUQy+ERFUHz6CTlszOBq7kl8O/WJ1pMKr1Rauas2DgSv4dPV+snViJKVUPrRYFFP1hx7EXr0aD63w5bV1E8nMzrQ6UuG1G0qDrD+olrSd5bsS3LdXSlU4WiyKyR4SQvgTT9DozzTCN+xj1s5Z7ncqa1oPxPgEMiLwZz5Zs9/qNEqpMkiLRQmo0r8/fo0a8dAv/kzb+C4nU09aHalwAiojLWLoxSrW/X6Qg6d0YiSl1MUKXSxExCYilTwRxluJjw81xv6dqgmpdF5/lrfj3rY6UuG1H4afI4W77Gv4dK2OF6WUuliBioWIzBKRSiISDGwHdonIWM9G8y4hN91EUKdODP7Vzrebv2TnqZ1WRyqcuh0h7BoeDv6FObEHdWIkpdRFCnpm0dwYkwT0Ab4F6gH3eSyVFxIRav5jLP7nMhi0zpeJ6yZ616O0ItB+GI3SdxCesofvtujESEqpCwpaLHxFxBdnsZhvjMkkn1FgK7qA5s2pHB1N93WZ7N+1nmUHllkdqXBaD8LY/Xg4ZCUz9Ua3UiqXghaL94B9QDCwQkTqAzprTj7Cx4zGbrPz0Opg3ox9k/RsL+oVHVwdaXYXd5oVbN1/jO2H9Z9YKeVUoGJhjPmPMaa2MaancdqPa7RYdTHfiAiqjRhOm7gkAn6P5+NtH1sdqXDaDSMgK4m7fGP5ZK2eXSilnAp6g3u06wa3iMgHIrIRuMXD2bxW9Qcfwl69OqNWVWLq5vdJSPGijm4NboCqDXg0dCXzfjtEcpoXdjJUSpW4gl6Gut91g7s7EA6MACZ4LJWXs4cEE/7E40TsOU3kzgze2viW1ZEKzmaDdkNpnBJHzcx4nRhJKQUUvFjkTHPaE5hujNlE/lOfKpcq/fvj17gxj6wMZNHv89l8fLPVkQquzb0gdp6o8iszV+/3rqe6lFIeUdBisUFEvsdZLJaISCjg8Fws75fTUS/46BlitgYxcd1EHMZL/spCr4Jr7qBn9k/sS0hk7Z9lZt4ppZRFClosHgCeAf5ijEkB/HBeilJXEHLjjQR16kTfldnsid/Eor2LrI5UcO2GEZBxiuiAOB0vSilV4KehHEAd4J8i8gbQ2RjjRddVrJHTUc8nOZWHfqvG5A2TScn0knGXrr4VKtXh0UqrWLz1KAnJaVYnUkpZqKBPQ00ARuMc6mM7MEpEXvVksPIioHlzKvfuTedfEzFHjvHB1g+sjlQwNju0HULjpHVcZRKYs14nRlKqIivoZaiewG3GmA+NMR8CPYA7PRerfAkfMxqb2PjbhquYsW0Gh88etjpSwbQdggB/C1vHrLUHdGIkpSqwwow6WyXXcuWSDlKe+V51FdVGDKfRukM0PuRg0oZJVkcqmCp14epu3JG1jGNnzvHjTi/qL6KUKlEFLRavAr+JyEciMgPYAPzLc7HKn5yOemNWV2XJn4uJPRprdaSCaTeUgNRj9AnZoeNFKVWBFfQG92dAJ+Br1+s6Y8xsTwYrb5wd9Z6gys7DdD9QhdfWv0a2wwuGAb/mDgiuwWOVVrLi9+PsO3HO6kRKKQtcsViISLucFxABxAMHgVqudaoQqvTvh1/jxgxbLvx+fDvz9syzOpJ7dl9ocw+NTq8iwnaaWet0YiSlKiJ3ZxZvXuH1hmejlT85HfV8D59gxO7a/Oe3/5CckWx1LPfaDUVMNs9GbGBO7EHSMr3gjEgpVaKuWCyMMTdf4aUDCRZByI03EnRdJ2778TTpiad4f/P7Vkdyr3pjaHAD3dOXciYlnUWbj1idSClVygraz6JvPq9bRaSGpwOWN86Oev9Aks7yjx2N+WTHJ+xP8oIbx+2HE3D2IP2r/qE3upWqgAoz3Mc04F7XayrwFLBKRHR61UIKaNaMyr170+yHvdRK9uWN9V5wRe/aXhBYlccqrSLuYCJbD52xOpFSqhQVtFg4gGbGmH7GmH5AcyAd6Ag87alw5Vn4mNGI3c4zG+uwPH45vx761epIV+YbAJGDaXD8J2r5ntPxopSqYApaLBoYY47lep8ANDXGnAJ0dpwiyOmoF7ZyB50Ta/Da+tfIcmRZHevK2g1FHJk8XzuOeXGHOJOq//RKVRQFLRa/iMhCERkmIsOABTjn4g4GEi+3k4j0EJFdIrJHRJ7JZ/twETkuInGu14O5tmXnWr+gsAfmDao/8CD26tV57JdA/kjcw5xdc6yOdGU1mkGdDnRLXUxaZjZfb4y3OpFSqpQUtFiMBKYDbYC2wAxgpDHmnDEm37m4RcQOTAHuwHnZarCINM+n6efGmDau17Rc61NzrY8u6AF5k5yOen5b/+De402YEjeFxLTL1t6yof0w/M/8wT01D/HJGp0YSamKoqA9uA2wEvgRWAasMO5/SnQA9hhj9hpjMoDZQO/ihC2PqvTvh9/Vjem95Aypacm8s+kdqyNdWYsY8K/EI6Er+eP4OVbvPWl1IqVUKSjoo7MDgXVAf2AgsFZE+rvZrTbO3t454l3r8uonIptF5EsRqZtrfYCIxIrIGhHpU5Cc3kh8fKg5dizEH2FsfGvm7JrDntN7rI51eX7B0Ko/9Y5+T93AdL3RrVQFUdDLUM/jnCVvmDFmKM6zhv/nZp/85ujOezbyDc6b561xnrHMyLWtnjEmCrgHmCwijS/5ApGHXQUl9vjx4wU8lLInuGtXgq7rRNuFuwnLDuS19a+V7cs77YYhWWn8s942lmw7xrEknRhJqfKuoMXCZozJPT71yQLsGw/kPlOoA1w0kYMx5qQxJt31dirQPte2w64/9wLLcd4rIc/+7xtjoowxUeHh4QU8lLInp6OeSUrmn7uasfrIan6O/9nqWJdXqw1ERHLzuW/JdjiYvU4nRlKqvCtosVgsIktcTy8NBxYB37rZZz3QREQaiogfMAjnU1TniUhErrfRwA7X+qoi4u9aDgO64Jyhr9wKaNaMyn36UHNRLO2z6/L6+tfJyM6wOtbltRuG34ntDKt/ms/WHSAr22F1IqWUBxX0BvdY4H2gNRAJvG+MuWJnPGNMFvA4sARnEZhjjNkmIuNEJOfpplEisk1ENgGjgOGu9c2AWNf6n4AJxphyXSwAwkePQux2xqwP50DyAWaBNin5AAAchklEQVTtmGV1pMtrNQB8g3goeAVHk9JYtkMnRlKqPJMyfW28EKKiokxsrJdMKHQFCW+9xcn/vcuXf49iUdBuFsYsJCwwzOpY+Zv3GGb7fLrxHhE1wvnkwY5WJ1JKFZKIbHDdH74id/NZJItIUj6vZBFJKrm4Kkf1Bx7EHhbGwKWppGel8fZvb1sd6fLaDUMyzvJ8/R2s3HOCvcfPWp1IKeUh7oYoDzXGVMrnFWqMqVRaISuSnI56jk3b+NvZ6/l699fsOLnD6lj5q9sBwq/lhuRF+NiET9fqxEhKlVcFvcGtSlGVfn3xu7oxHef9TphPFSasm1A2H6UVgXbD8D36G/c3OccXsQdJzdCJkZQqj7RYlEE5HfWyDhzkn0f/wsaEjSzZv8TqWPmLHAR2P0YE/kJSWhbfbD7sfh+llNfRYlFGBXftSnDn66g9ZxWRAVczKXYSaVllsPNbUDVoFs1V++bRItxXe3QrVU5psSijRIQaY8fiSEpi7PaGHDl3hI+2fWR1rPy1H4akneGZBr+zOf4Mmw6W8cEQlVKFpsWiDMvpqOf/9TL6Bnfhw60fcvTcUatjXarBDVCtEdclLiTIz65nF0qVQ1osyrjwMaPBbmfIChvZjmwmb5xsdaRLiUC7ofgcXM2DzbJZsOkwiSlluPe5UqrQtFiUcb41a1L9/hFkff8TT/j3YNHeRcQlxOXbdsTiEYxYPKKUE7pE3gM2H4b6/0x6loMvN+jESEqVJ1osvEC1+x/AHhZG17l/EB4QxsR1E3GYMjYWU2hNaNqDsD1f0bFeCJ+uPYDDUQYf91VKFYkWCy+Q01Ev/bdN/DP9Nrae3MrCvQutjnWp9sMh5QR/q7eHP0+c49c/dGIkpcoLLRZeokq/vvg3uZp6n/xMZJUWTN4wmZTMFKtjXazxLVC5LlGnvqFasB8z1+yzOpFSqoRosfAS4uNDjbFjyTxwgKcPRXI89TjTtkxzv2Npstmh7RBse5fzUCs7y3YkcORMqtWplFIlQIuFFwm+4QaCO1+H/4z59K3ZnRnbZhCfXMZuJLcdAiLc4/szDmP4TCdGUqpc0GLhRUSEGv/4B9lJSQzfEIrdZmfShklWx7pY5TpwdTcq7/ycW5pUY/a6A2TqxEhKeT0tFl4m4NprqRwTQ/rsrxlZoz9L9y9l/dH1Vse6WLthkHyEUfX2kZCcztLtx6xOpJQqJi0WXih89Ciw27l50SFqBddi4rqJZDvK0GivTW+H4Bq0TphH7SqBzFytPbqV8nZaLLxQTke9c4u/55ngvuw6vYuv93xtdawL7L7Q9l5k9/c83DaA1XtPsich2epUSqli0GLhpXI66jWcuYL2Ndrx343/JcuRZXWsC9oNBeOgv+1nfO3CJ2t0YiSlvJkWCy9lDwkmfNQTpG7cyNPnbiQxPZEj545YHeuCao2gYVeCt83izpY1+WpDPCkZZaiYKaUKRYuFF6vS19lRz/f9z+nfsDcJKQlla86LdsMg8QCP1o8nOT2LBXE6MZJS3kqLhRfL3VFvxO5a2MTGweSDZWcK1mZ3QWA1msZ/zbVXhTJzzf6yk00pVShaLLycs6NeZ1Lfn0GfdXAm4wzDFw9n16ldVkcDH3+IHIzsXMQD7ULZdjiJOJ0YSSmvpMXCyzk76o0lOymJHusd9I/15c8zfzJw4UBeWfMKZ9LPWBuw/TBwZBLNzwT72ZmpEyMp5ZW0WJQDOR31Kp3JpMsuG9/EfMOgawbx5e9fcufcO5mza451/TDCr4G6nfDf9Al929Zm4eYjnD6nEyN5q47T+9Fxej+rYxRbeTkOKL1j0WJRToSPHoURqHIqncr+lXm247PMuWsOTas2Zfya8QxaNIiNxzZaE67dUDi5m4fqHyUjy8EXGyrWeFHl6QeTqri0WJQTvjVrklzFj+CzWRx56SUc587RtGpTPuj+AW/c+AaJ6YkMWzyMZ355hmPnSnn4jRZ9wL8S9fZ9QYcG1XRiJKW8kBaLcuRMVT+SKvuS+Pkc9vaJIWX9ekSE2xvczvze83mk9SMs3beUu+bdxbQt08jILqXLQX7B0GoAbJ/P8HaV2X8yhV/2nLjiLne/t5q731tdOvmUUm5psShPREgMC6D+zI8B2D90GMcmTMSRlkaQbxCPt32ceX3mcV3Edby18S1i5sewIn5F6WRrPwyy0rgtewVhIX5ux4va5/cG+/zeKJ1sSim3tFiUQ0FRUTSaN5cqg+7m1Ecf8WfffqRu3gxA3dC6vHXLW7zX7T1sYmPkDyMZ+cNI9id5+CmliEiIaINv3EzujqrDjzuPcShRJ0ZSyltosSinbMHBRLz4InU/mIYjJYV9g+8hYfJkTIbz0lPn2p35Ovpr/h71dzYc20Cf+X3494Z/e3aq1vbD4NhWhtY/hQE+W6vjRSnlLTxaLESkh4jsEpE9IvJMPtuHi8hxEYlzvR7MtW2YiOx2vYZ5Mmd5MfuJFsx+osVF60K6dKHRgvlUjo7m5Lvv8efAu0nbuRMAX7svw1oMY2HMQu5seCcfbv2Qu+bexaK9izzT07plf/ANoubu2dx6bQ1mrz9IRpZOjKSUN/BYsRAROzAFuANoDgwWkeb5NP3cGNPG9Zrm2rca8CLQEegAvCgiVT2VtbyzV6pErVf/RZ133iHrxAn+HDCQE+++i8lyDuwXFhjGK9e/wic9PyE8KJxnfnmGYYuHsePkjpINElAJWvaFLV8xtH11TpxNZ8m2oyX7HUopj/DkmUUHYI8xZq8xJgOYDfQu4L63A0uNMaeMMaeBpUAPD+WsMEJvuZlG3yyg0m3dOD75LfYNvof0vXvPb48Mj2TWnbN4ufPL7E/az6BFgxi/ejyJaSU4REe74ZB5juvTVlC3WqD26FbKS3iyWNQGcve+inety6ufiGwWkS9FpG5h9hWRh0UkVkRijx8/XlK5yzWfqlWpPWkStSe9SeaBA/wZ05eTH32EcTgvB9nERt8mffkm5hvuufYevtr9FXfOvZPZO2eXzHwZdaIgvBm2jTMY0rE+6/48xe/HdGIkpco6TxYLyWdd3gvh3wANjDGtgWXAjELsizHmfWNMlDEmKjw8vFhhK5pKPXvSaOE3BHfpQsKEiRwYOoyMgxfqcyW/Sjzd4Wm+vOtLmlVrxv+t/T8GLRxE7NHY4n2xiPNG9+GNDKp3Bj8fG5/o2YVSZZ4ni0U8UDfX+zrARRMaGGNOGmPSXW+nAu0Luq8qPp/wcOpMeZuIV18lbedO9vbuw+nZsy+6uX111auZ2n0qk26aRFJGEiOWjOAfP/+Do+eKca+h9d1g96fyjs/o1SqCrzce4ly6ToykVFnmyWKxHmgiIg1FxA8YBCzI3UBEInK9jQZy7qguAbqLSFXXje3urnXqCqb3mM70HtMLtY+IUCWmD42+WUBQmzYcfellDj7wIJlHjlzU5rb6tzG/z3wejXyUHw/+SPS8aKZunkp6dvoVPv0ygqpB82jY/DlDompyNj2LeXGHCv85SqlS47FiYYzJAh7H+UN+BzDHGLNNRMaJSLSr2SgR2SYim4BRwHDXvqeA8TgLznpgnGud8hDfiAjqfjCNq156kZS4OPbeFU3i3HkXnWUE+gTyWJvHmN9nPl1qdeE/v/2HmPkxLD+4vPCP2rYbBmlnaJu8nOYRlZi5WidGUqos82g/C2PMt8aYpsaYxsaY/3Ote8EYs8C1/KwxpoUxJtIYc7MxZmeufT80xlztehXu12VVJCJC1UGDaDRvLv7XXsORZ58lfuTjZOV5eKB2SG3+ffO/ee+29/C1+fLEj0/w6A+Psu/MvoJ/WYProVpjZOPHDOlUn51Hk9l44HTJHpBSqsRoD251Cb969ag/YwY1nn6acytXsveuaJIWL76kXedanfky+kvGRo1lU8ImYhbEMGnDJM5lnnP/JSLOocsP/EqfumcJ9fdxO16UUso6WixUvsRup/qI4TSc+zW+detyaMyTHHrqKbJOX/zbv6/Nl6EthvJNzDf0atSL6Vun02tuL7754xv3l5Xa3AM2H4K2fEq/9nX4dstRTp4twj0QpZTHabFQV+TfuDENPptF+JgxJC1dxt67okn+8adL2oUFhjG+y3hm9ZxFRHAEz618jqHfDWX7ye2X//CQGnDNHbDpM4ZE1SQj28Gc2HgPHo1Sqqi0WCi3xMeHsL8+QsMv5uBTvTrxjz3G4WefIzv50s50rcJb8UnPTxjXeRwHkg8waOEgXl79MqfTLnM/ot1wSDnJ1ad/oVOjasxat59snRhJqTJHi4UqsIBrr6XhF3Oo/tdHOLNgAXuje3Pu118vaWcTGzFNYlgYs5AhzYcwb/c87px7J7N2zLq0F3jjm6FyXdgwgyGd6nPwVCorftfe+EqVNVosVKGInx81xoyhwWezsAUGcuD+Bzg6bhyOc5fe1A71C+Uff/kHX0Z/SYvqLXh13asMXDiQ9UfXX2hks0Pb+2DvT3SPSCc81F/Hi1KqDNJioYoksHVrGn79FdWGD+f0Z7Od07jG5j8USOMqjXn/tveZfNNkUjJTuH/J/fz9579f6AXedgiIDb/NnzD4L3X5aVcC2ZkhpXg0Sil3tFioIrMFBFDzmacvTON631COTXwNR1raJW1FhFvr38q83vN4rM1jLD+4nLvm3sV7m94jPSQMrr4N4j5lUFQtBEhJvKaUj0YpdSVaLFSxXTSN6/Tpzmlct2zJt22ATwCPRj7Kgj4LuKHODbwd9za95/Xmx4ZRmOQj1Er4hW7NapJ6pinGof95KlVW6P8bVYm4ZBrXQYNJeOut89O45lUrpBaTbprE1O5TCbAHMHr3TB6tXYe9G95nSKf6mOxA0s42KN2DUEpdlo/VAVT5kjON67FXJ3Dyf+9y9qfl1Jo4gYBr8r+s1CmiE19Ef8HnOz/nndhJ9Mvcwz0Jb2P3a0DysU70fnslAb52Av3sBPo6XwG5lgP97M7tvnYC/WzO7bm2nX+fa9luy28EfKXUlWixUCUuZxrX0Nu6ceSFF/mz/wDCR46k+oMPID6X/ifna/NlSPMh3FG1Of/5qh8z//iakIZ2THJz0oOv4Ww2ZGVAVqohMxvnK8uQkWUwxoZBAAFjc/6JgJHzyyZnvRHAho/dRoCPHT+7D/4+Pvjb7fj7upZ97AT4+BLg4yws/r6+BPr4EODrQ6Cv608fHwJ9fQn0c64L8vMjyLUc7OdcH+Djg91mx4YN45qKJdORCQZMzv/MhT8BHMZx0TbgQps87Q0Gh3HgcBiyjcFhLrx3GEO2cbjeOz/XgXG1dWBMrjYOh2tfVztjcGAwruVshzm/3hhDeoYvAF9sXeH8x8vVJcZw6aQzF/fiN/lsz713/tsu3evC9tzb8hsxwORpnyMtPQAwfLjh+3w/25ukpfsjpfD7j5SXkT6joqJM7GWexlHWyTp9mmPjx5P07XcEtG5NrQmv4t+o0eV3mBHN1jN/cn8opOpFUqUKxJF2FdseWVqkfUVkgzEmyl07PbNQHpUzjWtot24cfXkcf8b0JfzJMVQbOhSx5VMN2g+j5Zf3E01N4gICmXn/GowxZJts52/eruXzv1nn98L1GzO52rt+887ZN9tx4TNyb7/oM1zL2Q4H6VlZpGVlkZ6VTXpWJulZznXpWVlkZGWTkZ1NenYWma7ljOwsMrOdy9uO7wGEptUaINgQAUEQkfN/2s4vX9huy2nj2mYTm2sZRGzYBGw5n+faz/n+wr421/65P8smzs+y5fl+53vX57i221zfI2JDgGlxcxDgkXaDL/ybycWLkneiS7n4Td5fgiXXr8WXbrt4S9695eJG+U6xKXm+H2Dy2g8BeKrTA/ns4V0mrfkAn1I4s9BioUpFpZ49CfrLXzjywoskTJjI2WU/EPHqv/CrW/fihtf2gsBq3JqSzOaAIAJ9Aq0JXII6Tu8HwLx7X7I2SAn4dNdHADzWqae1QYrpf5veAWBo21ssTlJ8U+KmlMr36Im+KjU+4eHUeWdKnmlcP7/4WrOPP7S5h44pKTw8N9W6sEqpi2ixUKXq0mlcX+Lggw9dNI0r7YZis0HtGinWBVVKXURvcCvLGGNI/Pxzjr32OmK3U/P556jcuzciwqknw6lWOQOCwqBSrYtfoTnLtaFSBPiHWn0oV/Rtt+YA9Fx2heHavUR5OZbychxQ/GPRG9yqzMuZxjW4c2cOP/ccR555luSly4h4+SW27KlCRFgqTfv2gqTDcOYQHFwHqflMxe5fKZ9CkquYVKoNgVXz3ulUShWCFgtluZxpXE99PJPj//43e3vdhbH5sCe1Ek3veuvixpmpkHzEWUAueh1y/pmwA5KPcslz+z4BEOoqHHnPVHKKTEgN5yi4SqlLaLFQZULONK4hXW/g8DPPUnNLImlnDQcffQxbYCC24CAkMBBbYBC2oCDnuqBAbEFNkcBIbNWDne8DA7H5+yHmLLbM00hqApJ8BJJzFZaDa50FJzsjbwhXQYnIdWZS6+IiExoBPn7W/CUpZSEtFqpMyZnG9dfOrQhKg8yjRzEpKThSUnCkpuJITYXs7IJ/oN3uLCCBgdiCgpCgIGyBrbAFdcDma8fmYxCfbGySgY00bIkp2BzJSPZmbJnLnet8DDYfh6utwVapOrZqEUjl2vlf8gqNAH8dYl2VL1osVJkjPj6cqSScqQRt53590TZjDCYjA0dKCiY19UIRSUnFkXLOuS41Fce5nOKScqHtuQsFJzspmazUFNd+zvXmkqHVg12v/JxA7Cew+cQh9mxXQcn18vNBggKxBYfQ1JaKAzg+qjfnu43l6WyWdxHk0jZ577m4eX+hs5tcpo2b9bZL2zWSNAxw4rmheLOG4vy39vbjAOexZJXCeGdaLJRXERHE3x+bvz9UrVqin22ys3GkpmFScwqQq5jkLjjni0uudWfP4kg6ieNsEo5zyWSeO4s5l4bjZAaOjEQcGf5ghBMHfi/RvNbwB+D4wfVu2pV15eU4APwJrJLp8W/RYqHKpHQp/Z7bYrdjDwmGkMudTRRNYR9tzPdx9ss94l6Y9bnXnR+lz+S73uDIs4/zz++jOwDQfd6a/L/XS3zfpxPg/ccBOcfiy9Ue/h4tFkqVMZLfI76l/Njv5b7N4dpiC6lcemE8oLwcB1w4Fk/THtxKKaXc0jMLVSZNuKcBADHWxlBKueiZhVJKKbe0WCillHLLo5ehRKQH8BZgB6YZYyZcpl1/4AvgL8aYWBFpAOwAdrmarDHG/NWTWZXylJfvbQaAd88A4VRejqW8HAeU3rF4rFiIiB2YAtwGxAPrRWSBMWZ7nnahwChgbZ6P+MMY08ZT+ZRSShWcJy9DdQD2GGP2GmMygNlA73zajQdeA/J2n1VKKVVGeLJY1AYO5nof71p3noi0BeoaYxbms39DEflNRH4WkRvy+wIReVhEYkUk9vjx4yUWXCml1MU8WSzy6ylyvquoiNiAfwN/y6fdEaCeMaYt8BQwS0QqXfJhxrxvjIkyxkSFh4eXUGyllFJ5efIGdzxQN9f7OsDhXO9DgZbAcleP1auABSISbYyJBdIBjDEbROQPoCmgU+FVEM0jLvndQCllIU+eWawHmohIQxHxAwYBC3I2GmPOGGPCjDENjDENgDVAtOtpqHDXDXJEpBHQBNjrwaxKKaWuwGNnFsaYLBF5HFiC89HZD40x20RkHBBrjFlwhd27AuNEJAvIBv5qjMlnPk1VXk3vMd3qCEqpXDzaz8IY8y3wbZ51L1ym7U25lr8CvvJkNqWUUgWnPbiVUkq5pcVCKaWUW5LvRCteKCoqysTG6sNSSilVGCKywRgT5a6dnlkopZRyS4uFUkopt7RYKKWUckuLhVJKKbe0WCillHJLi4VSSim3tFgopZRyS4uFUkopt7RYKKWUcqvc9OAWkePA/mJ8RBhwooTiWKm8HAfosZRV5eVYystxQPGOpb4xxu3sceWmWBSXiMQWpMt7WVdejgP0WMqq8nIs5eU4oHSORS9DKaWUckuLhVJKKbe0WFzwvtUBSkh5OQ7QYymrysuxlJfjgFI4Fr1noZRSyi09s1BKKeWWFgsXERkvIptFJE5EvheRWlZnKioReV1EdrqOZ66IVLE6U1GJyAAR2SYiDhHxuidXRKSHiOwSkT0i8ozVeYpDRD4UkQQR2Wp1luIQkboi8pOI7HD9tzXa6kxFJSIBIrJORDa5juVlj32XXoZyEpFKxpgk1/IooLkx5q8WxyoSEekO/GiMyRKRiQDGmKctjlUkItIMcADvAX83xnjNdIgiYgd+B24D4oH1wGBjzHZLgxWRiHQFzgIfG2NaWp2nqEQkAogwxmwUkVBgA9DHG/9dRESAYGPMWRHxBVYCo40xa0r6u/TMwiWnULgEA15bRY0x3xtjslxv1wB1rMxTHMaYHcaYXVbnKKIOwB5jzF5jTAYwG+htcaYiM8asAE5ZnaO4jDFHjDEbXcvJwA6gtrWpisY4nXW99XW9PPKzS4tFLiLyfyJyELgXeMHqPCXkfuA7q0NUULWBg7nex+OlP5TKKxFpALQF1lqbpOhExC4icUACsNQY45FjqVDFQkSWicjWfF69AYwxzxtj6gKfAo9bm/bK3B2Lq83zQBbO4ymzCnIsXkryWee1Z6zljYiEAF8BY/JcWfAqxphsY0wbnFcQOoiIRy4R+njiQ8sqY0y3AjadBSwCXvRgnGJxdywiMgzoBdxqyviNqUL8u3ibeKBurvd1gMMWZVG5uK7vfwV8aoz52uo8JcEYkygiy4EeQIk/hFChziyuRESa5HobDey0KktxiUgP4Gkg2hiTYnWeCmw90EREGoqIHzAIWGBxpgrPdVP4A2CHMWaS1XmKQ0TCc552FJFAoBse+tmlT0O5iMhXwDU4n7zZD/zVGHPI2lRFIyJ7AH/gpGvVGi9+sisG+C8QDiQCccaY261NVXAi0hOYDNiBD40x/2dxpCITkc+Am3COcHoMeNEY84GloYpARK4HfgG24Pz/O8BzxphvrUtVNCLSGpiB878vGzDHGDPOI9+lxUIppZQ7ehlKKaWUW1oslFJKuaXFQimllFtaLJRSSrmlxUIppZRbWiyUKgQROeu+1RX3/1JEGrmWQ0TkPRH5wzVi6AoR6Sgifq7lCtVpVpVtWiyUKiUi0gKwG2P2ulZNwzkwXxNjTAtgOBDmGnTwB+BuS4IqlQ8tFkoVgTi97hrDaouI3O1abxORd1xnCgtF5FsR6e/a7V5gvqtdY6Aj8E9jjAPANTrtIlfbea72SpUJepqrVNH0BdoAkTh7NK8XkRVAF6AB0AqogXP46w9d+3QBPnMtt8DZGz37Mp+/FfiLR5IrVQR6ZqFU0VwPfOYa8fMY8DPOH+7XA18YYxzGmKPAT7n2iQCOF+TDXUUkwzU5j1KW02KhVNHkN/z4ldYDpAIBruVtQKSIXOn/g/5AWhGyKVXitFgoVTQrgLtdE8+EA12BdTinteznundRE+fAezl2AFcDGGP+AGKBl12joCIiTXLm8BCR6sBxY0xmaR2QUleixUKpopkLbAY2AT8C/3BddvoK5zwWW3HOG74WOOPaZxEXF48HgauAPSKyBZjKhfkubga8bhRUVX7pqLNKlTARCTHGnHWdHawDuhhjjrrmG/jJ9f5yN7ZzPuNr4Fkvnn9clTP6NJRSJW+ha0IaP2C864wDY0yqiLyIcx7uA5fb2TVR0jwtFKos0TMLpZRSbuk9C6WUUm5psVBKKeWWFgullFJuabFQSinllhYLpZRSbmmxUEop5db/B2vJXammORGiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores =  np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'logloss' )\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的logloss。在测试集上当C=1时性能最好（L1正则）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 换正确率做评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7747395833333334\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "\n",
    "#缺省scoring为正确率\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5)\n",
    "grid.fit(X_train,y_train)\n",
    "\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00177946, 0.00259829, 0.00234981, 0.00335908, 0.00301428,\n",
       "        0.00251174, 0.00243735, 0.00248222, 0.00247135, 0.00246506,\n",
       "        0.00247321, 0.00250254, 0.0025743 , 0.00247312]),\n",
       " 'std_fit_time': array([1.53982569e-04, 1.54964814e-04, 5.87787059e-04, 4.56830156e-04,\n",
       "        7.00717913e-04, 8.05393346e-05, 6.71817698e-05, 4.09376295e-05,\n",
       "        1.45643008e-04, 2.41335195e-05, 4.46843997e-05, 5.19105964e-05,\n",
       "        1.60209948e-04, 4.72201223e-05]),\n",
       " 'mean_score_time': array([0.00059314, 0.00076575, 0.00066986, 0.00092306, 0.00072503,\n",
       "        0.0004797 , 0.00047946, 0.00046868, 0.00047712, 0.00046878,\n",
       "        0.00048728, 0.00051818, 0.00054264, 0.00047712]),\n",
       " 'std_score_time': array([5.28154790e-05, 1.40210997e-04, 1.33156860e-04, 2.67499211e-04,\n",
       "        2.52561967e-04, 1.94529691e-05, 2.36109038e-05, 2.48870627e-06,\n",
       "        9.84111494e-06, 4.91720618e-06, 1.89018875e-05, 5.13537459e-05,\n",
       "        8.44828422e-05, 1.11577857e-05]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
       "                    100, 1000, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'split0_test_score': array([0.64935065, 0.73376623, 0.68181818, 0.75324675, 0.75974026,\n",
       "        0.76623377, 0.75974026, 0.75974026, 0.75974026, 0.75974026,\n",
       "        0.75974026, 0.75974026, 0.75974026, 0.75974026]),\n",
       " 'split1_test_score': array([0.64935065, 0.71428571, 0.68181818, 0.70779221, 0.74025974,\n",
       "        0.74675325, 0.74675325, 0.74025974, 0.74025974, 0.74025974,\n",
       "        0.74025974, 0.74025974, 0.74025974, 0.74025974]),\n",
       " 'split2_test_score': array([0.64935065, 0.75324675, 0.73376623, 0.77272727, 0.77272727,\n",
       "        0.79220779, 0.78571429, 0.78571429, 0.78571429, 0.78571429,\n",
       "        0.78571429, 0.78571429, 0.78571429, 0.78571429]),\n",
       " 'split3_test_score': array([0.65359477, 0.78431373, 0.7254902 , 0.80392157, 0.78431373,\n",
       "        0.79738562, 0.79738562, 0.79738562, 0.79738562, 0.79738562,\n",
       "        0.79738562, 0.79738562, 0.79738562, 0.79738562]),\n",
       " 'split4_test_score': array([0.65359477, 0.75163399, 0.7124183 , 0.77124183, 0.75816993,\n",
       "        0.77124183, 0.77124183, 0.77124183, 0.77124183, 0.77124183,\n",
       "        0.77124183, 0.77124183, 0.77124183, 0.77124183]),\n",
       " 'mean_test_score': array([0.65104167, 0.74739583, 0.70703125, 0.76171875, 0.76302083,\n",
       "        0.77473958, 0.77213542, 0.77083333, 0.77083333, 0.77083333,\n",
       "        0.77083333, 0.77083333, 0.77083333, 0.77083333]),\n",
       " 'std_test_score': array([0.00207782, 0.02321726, 0.02172389, 0.03153907, 0.01483053,\n",
       "        0.01836721, 0.01800398, 0.01992491, 0.01992491, 0.01992491,\n",
       "        0.01992491, 0.01992491, 0.01992491, 0.01992491]),\n",
       " 'rank_test_score': array([14, 12, 13, 11, 10,  1,  2,  3,  3,  3,  3,  3,  3,  3],\n",
       "       dtype=int32),\n",
       " 'split0_train_score': array([0.6514658 , 0.75081433, 0.71335505, 0.76710098, 0.76547231,\n",
       "        0.77035831, 0.77035831, 0.77198697, 0.77361564, 0.77361564,\n",
       "        0.77361564, 0.77361564, 0.77361564, 0.77361564]),\n",
       " 'split1_train_score': array([0.6514658 , 0.75732899, 0.71335505, 0.77361564, 0.7752443 ,\n",
       "        0.78338762, 0.77850163, 0.77850163, 0.77850163, 0.77850163,\n",
       "        0.77850163, 0.77850163, 0.77850163, 0.77850163]),\n",
       " 'split2_train_score': array([0.6514658 , 0.74592834, 0.70032573, 0.76058632, 0.77361564,\n",
       "        0.77035831, 0.7752443 , 0.77198697, 0.7752443 , 0.77361564,\n",
       "        0.77361564, 0.77361564, 0.77361564, 0.77361564]),\n",
       " 'split3_train_score': array([0.6504065 , 0.73333333, 0.70243902, 0.75609756, 0.75934959,\n",
       "        0.76422764, 0.76747967, 0.76747967, 0.76747967, 0.76747967,\n",
       "        0.76747967, 0.76747967, 0.76747967, 0.76747967]),\n",
       " 'split4_train_score': array([0.6504065 , 0.74634146, 0.70569106, 0.76260163, 0.77560976,\n",
       "        0.77723577, 0.78211382, 0.77886179, 0.77886179, 0.77886179,\n",
       "        0.77886179, 0.77886179, 0.77886179, 0.77886179]),\n",
       " 'mean_train_score': array([0.65104208, 0.74674929, 0.70703318, 0.76400042, 0.76985832,\n",
       "        0.77311353, 0.77473955, 0.77376341, 0.77474061, 0.77441487,\n",
       "        0.77441487, 0.77441487, 0.77441487, 0.77441487]),\n",
       " 'std_train_score': array([0.00051895, 0.00786489, 0.00543745, 0.00596881, 0.00641678,\n",
       "        0.00658318, 0.00530451, 0.00434145, 0.00413207, 0.0041437 ,\n",
       "        0.0041437 , 0.0041437 , 0.0041437 , 0.0041437 ])}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8VVW2wPHfyk0PLYQEgVBClQ4SCIrggCiIiqCjgmOdgr4nPnUcrDOoOGMZx3GaM6hYsIyI2BhEEERFGYEA0pt0QksICRBCSLnr/XFvYogJuSQ5OSnr+/ncT07Z+9x1KGfl7H3O3qKqGGOMMWcS5HYAxhhjaj5LFsYYY8plycIYY0y5LFkYY4wplyULY4wx5bJkYYwxplyOJgsRGSkiW0Rkm4g8WMr+NiLyhYh8JyJrRWRUsX0P+ettEZERTsZpjDHmzMSp9yxExANsBS4BUoBkYLyqbixW5iXgO1X9l4h0A+aqajv/8jvAAKAlsBDorKoFjgRrjDHmjJy8sxgAbFPVHaqaC8wAripRRoFG/uXGwH7/8lXADFU9pao7gW3+4xljjHFBsIPHbgXsLbaeAiSVKPMY8JmI3AVEAcOL1V1aom6rM31Zs2bNtF27dpUI1xhj6p+VK1ceVtXY8so5mSyklG0l27zGA6+r6nMicj7wpoj0CLAuIjIBmADQpk0bVqxYUcmQjTGmfhGR3YGUc7IZKgVoXWw9nh+amQr9ApgJoKrfAuFAswDroqovqWqiqibGxpabGI0xxlSQk8kiGegkIgkiEgqMA2aXKLMHuBhARLriSxZp/nLjRCRMRBKATsByB2M1xhhzBo41Q6lqvohMBOYDHuBVVd0gIlOAFao6G7gPeFlE7sXXzHSr+h7P2iAiM4GNQD5wpz0JZYwx7nHs0dnqlpiYqNZnYYwxZ0dEVqpqYnnl7A1uY4wx5bJkYYwxplyWLIwxxpTLkoUxxphyWbKoQ26bdxu3zbvN7TCqRF06F2PqAksWxhhjymXJwhhjTLksWRhjjCmXJQtjHFaX+l+SXruGpNeucTuMSqsr5wHVdy5OjjprjAE2HjjmdgjGVJrdWRhjjCmXJQtjjDHlsmRhjDGmXJYsjDHGlMuShTHGmHJZsjDGGFMuSxbGGGPKZcnCGGNMuSxZGGOMKZejyUJERorIFhHZJiIPlrL/eRFZ7f9sFZHMYvv+KCIbRGSTiPxNRMTJWI0xxpTNseE+RMQDvABcAqQAySIyW1U3FpZR1XuLlb8L6OtfvgAYBPTy7/4GuAj40ql4jTHGlM3JO4sBwDZV3aGqucAM4KozlB8PvONfViAcCAXCgBDgkIOxGmOMOQMnBxJsBewttp4CJJVWUETaAgnAIgBV/VZEvgAOAAL8Q1U3lVJvAjABoE2bNlUavDFVpW3edrdDMKbSnEwWpfUxaBllxwGzVLUAQEQ6Al2BeP/+BSIyRFUXn3Yw1ZeAlwASExPLOraphcb9fYNvYaS7cRhjfJxMFilA62Lr8cD+MsqOA+4stj4WWKqqWQAi8ikwEFhcSl1jTDV59G3/DX4tn56jrpwHVN+5ONlnkQx0EpEEEQnFlxBmlywkIl2AaODbYpv3ABeJSLCIhODr3P5RM5Spu7JzC8jOLXA7jCpx68xcbp2Z63YYxlSKY8lCVfOBicB8fBf6maq6QUSmiMjoYkXHAzNUtXgz0ixgO7AOWAOsUdX/OBWrMcaYM3N0pjxVnQvMLbFtcon1x0qpVwDc7mRsxhhjAmfTqpoaySMnyQ6DAm8BniCP2+FUnNdLWEgBHo9Ceu1/KioyPN+3kLEbJMj3CfL8sFza50f77f3a2siShakxCrwFJB9KZu6OuXx6bRA5ocJTb51Hs/BmxEXGFX2aRzU/fT2yOVEhUe4Ffuq47+KZsQsy/T8LP5l7uDgpx1fu7+cFdDhVUC+ggnpB/T8pvlxsH15Qr/jrCWjx9VKOU0bd4scv2leiTgd/Y/Hen15UiT8w+eGnyA/rUnI7pewvq0xZxyt9ufOpU77zuKrUp/lrlc6nTuENdf5hUEsWxlWqyob0DXyy4xPm7ZrH4ZOHiQqJovseJT5daf6rCaRmp5Kancqe43tYcXA5J3KOE1wAQV4I9vp+NgqKJDYshrjQpsSGNiUmpAkxIU1oGtKYpsGNiQ5uSENPFEEFXrSgAM3Lh4J8ND8fzS9A8/PAv10LSqzn5UJ2JnoiA83OQLOPQvYxNOc4mpMFebn+C6z/Yk0w6glHg8JQ6cDJtMOoVwhv1tx3zAJv0af4Ov7YynzA3ElBgniCEI8HPEH+5SDweIqWJSiIk0d878ZGxLUo/Bv84WdR3Frsh5bYVvzkSmwrLF9qveLbih/nDN+Pt5SYfD/1lC9x5B05UcofRu2ipwTxWLIwddTOozuZu3Muc3fMZc/xPYQEhTA8ZhBXSgc67DzFoY9fJyQfQpLfRPPzId93EcfrLeOIx/2fXaXuPeb/VIgoIiBB6vvlNQjfhTQ4GAlugISGIiFhEB6BhEYgoeEQ7EGCQwjyeDh5aAkKNO5+HuLxICHB4AlGgoORYI/vOP51Xz3/ekiw72IdHIL4t5daLyTYd5EP9u8rvly0XvwYHiQkxBdL4XpQYM+6zB3eDYBR87+s6J9mjVB0HgvXuxxJ5RWeSyeHv8eShak2h04cYt6ueXyy4xM2HdlEwxzhyqyODEkbSPOth8nbvAi8CzkaEgICJyKg7ZgxpV5US14sS15UNSiILM0hM/8YmflZZOQeIT37AEdyUknPPcLh/GOkaQ7HgrwUBOH7eKBAIFy8xAAxwWE0C2tE88g44hq2Ii66A82bnUtcsx40jYorty/lZG4Bq/Zk8PaUJzgVFEbXUb8gPCSI8BAPYSEewoN9y76PfznYU6zMD9tCPIKNpWncZMnCOOroqaMs2L2AuTvnsmVHMufu9TIqLYYH9jYlcnca6CYkNJSQ3r1pfMcdRA7oT0Tv3sy7sh8ASY88HNgXqUJW6ml9Bo0zdtGqsC/h2D5Oa8bwhEF0W7Ibtya1cXNSI5twKCySVE8QqeSTeiqT1OxUtmcf4vCJ7ynI2gwHPi+qHizBxETE0Dzyh/6TJmHNyDoRxYH0UL7f72FzipCXH4K0GE6oN48vvtyGt4KtBUHCD4kluFjCCQk6LcEUJp6w4JJJqMT+Eokp/LRjeQgLDiIoyJKT+YElC1PlTuaf5Ku9X7FozQccW7aUc3fnc9O+EM455HuSRsKPE9G3D5Gjryeqf3/Ce/UiKCys/APnnoDMPad3IBfvWM7LPr18wxbQpC20uxCi2/k/bX0/G5wDQUFEAu38n7IUeAtIz0knNTuVQ9mHivpQUo4fZMeRfSzP2MTx/G9QyfmhUgSEd4JoTwMapGcRnq80Prenr19DwauKV319NqcvF+7TonLFyxeu56iSXbJ+juLN8Zfx+rcF+HdWGhHfi1gSJASJECTgHdsCASa/emUljuy+grG+Ppfafh7gO5cGWRGMcvh7LFkAt83zvSf/2sjXXI6k9srz5rF87adsWDiT/FVr6bwrj9uO+HdGhBPVL5HIG/oT2b8/ET26I6GhZzxeZHg+reKy4f1f/ZAYTqSeXigkynfhb9oeOgw7PRk0aQMhEVVybp4gD3GRcYTShP2HmrFtZwuW7Uxn4/5jeBVCPUH0bt2Y89pG0rFlAXHRORzN8yeXE4dYt/Md8jzQOKxxlcRzVvwJpsD7QxLyeqGgaNn3s0ApWi5tf+G2k/n5gBAW1KD6z6UK5ebnARBay88DfOcSluf8XaAlizqkugffO5WSwuZF77Pv6wVErd9JXIaXwUBuRAhBvXsTe+FwogYMILxbN19HaqDyc0nslk5URD7sWepLAJ1HFLs78H8iYxx9Zj896xTLdx5h2c4jLN2RzpZDx1GFsOAg+rZpwl3DOpHUvinntYkmPKTs/ou5v30bgFEPT3Us1uqS9No1ACy59Z1yStZsdeU8wHcuxxs5/z2WLExAVJW8vXvJTk7m4Defk52cTMThLEKB2HBI79KcY9ddSLfh19KgWw/fEzgVtfSfNIjMJ3lDDP3fW1dl51Ce1OM5LNtxhGU701m24wjfp2YBEBHioV/baC7v2YKk9jH0bt2YsODAz+/xn3UFcLyZwBgnWbIwpVJVcnfuIjs5mezkZI4vX4qmHgbgaCRsbh3EyZ90pP1PrmTQ4PE0CGtYNV987AAsfpZD6eGkZYRXzTHLcODoydOSw47Dvmfuo0I9JLZrytjzWpGUEEPPVo0JDbbp6k39ZsnCAP7ksG0bJ/zJIXvFCgrSfMkhq2EIa+Lz2dg3CO3TjaSBVzM6YQRNw5tWfSALH4WCPDbtbFLlh957JJtlO4+wbEc6y3YeYc8RX4d4w/BgBrRryrgBrUlKiKF7y0YEeyw5GFOcJYt6Sr1eTm3dSvbyYskhIwOAvGaN2Z4QxteJwaxvrTTo0IFR7S/nvoTLaNWglXNB7VkKa9+Fwb8h+6vKtSWrKrvTs4vuGpbtPMK+zJMANIkMYUC7ptxyQTuSEprStUUjPPaYqDFnZMmintCCAnI2bS5qVspeuRLv0aMABLdqyfHEzqxocZKPG25jb8MsWjZozKj2v+SOhMvoHN3Z+QC9BTD3N9AoHgb/mte3vw8E3s6vqmxPO1EsOaRz6Jhv/J+YqFCS2jdlwpD2JLVvSue4hvYOgTFnyZJFHaV5eeRs3Eh2cjInkpM5uXIV3ixfh21I2zY0GH4xhzo3Y2HT/XyUtYTjuSuJDotmRLurubz95fSO7V29bwyvfB0OroOfvgah5Q8K6PUq36dmnXbncDjLlxziGoaR1D6GpISmDGzflA6xDeztZ2MqyZJFHeLJ9xJ1PI89v/gl2d99h2b72uRD27en0eWXE9k/kX0dG/PR8WV8uutTUrNTiTwWycVtLmZU+1EktUgiJCik+gPPPgKLnoB2g6H72FKLeL3KpoPHiu4alu88Qka271n5lo3DGdypGUkJTUlqH0O7mEhLDsZUMUsWdcCpbdtIn/YKLXefQID81FSajBlD5ID+RCYmsi/0BB/tnMvcHS+ya9kugoOCGdxqMJP6T+Ki+IuICK6al9cqbNHvIecYXPZM0XsTqsKJ3Ja8vHhHUXI4luN7A7x10wgu7trcf+cQQ3x0hCUHYxxmyaIWy171HenTppG1aBESEUFW4xCONQ5l+H9mk5adxse75jF36V2sT1+PIPQ/pz+3dr+V4W2Hu/M2cWkOrIWVr0H/X0Hz7kWb1x2+jZzjHdi4bxMJzaIY1bMFSe2bkpQQQ8smLic3Y+ohR5OFiIwE/gp4gGmq+nSJ/c8DQ/2rkUCcqjbx72sDTANa4xviZpSq7nIy3tpAVcn66ivSX57GyZUr8TRuTLM77yT6xp/xya2XsrZ1ATM++yXLDyxHUbrFdOM3ib9hZLuRNI9q7nb4p1OFT++HiGgY+lDR5k/WHiDneAcio9fxxf/cQ/NGzr5vYYwpn2PJQkQ8wAvAJUAKkCwis1V1Y2EZVb23WPm7gL7FDvEG8AdVXSAiDYCyJjKoFzQvj2Nz55I+7RVOff89wS1b0Pzhh2l8zdWsztrMXzb8iU9H55AXDG2yDnBH7zu4LOEyEhonuB162da9B3u+hSv/5ksY+IbYmPzxeoLD02gQu8IShTE1hJN3FgOAbaq6A0BEZgBXARvLKD8eeNRfthsQrKoLAFQ1y8E4azRvdjaZs94n/fXXyN9/gLBOnWj5zNPkDk3iw92f8NFn49h1bBdRIVH02+2h/85gfvXGnJrfhn/qOHz2O2jZF/reVLT5sf9s5FhOHo1bf42IG1PGGWNK42SyaAXsLbaeApQ64a2ItAUSgEX+TZ2BTBH5wL99IfCgqhY4F27Nkp+RQcZbb5Px9tsUZGYSkdiPuN/9llXt4fntH/L1R49SoAWcF3cev+j5Cy5teylfj/kJQM1PFACL/wRZB2Hc2+CfpW3e+oP8Z81+7rukM2/tyXQ5QGNMcU4mi9KuWGX9qjgOmFUsGQQDg/E1S+0B3gVuBV457QtEJgATANq0aVP5iGuAvH37SH99OpmzZqEnT9Jg2DByx1/Oh5FbmL39CQ7vOUxMeAw3d7+ZsR3H1uxmprIc3gbfvgB9fgbxiQBknMjltx+tp3vLRtzxkw689YbLMRpjTuNkskjB1zldKB7YX0bZccCdJep+V6wJ6yNgICWShaq+BLwEkJiYWKvbLHK2bCX9lWkc+2QuiBA1aiQbRnZixqklrNrxAB7xMDh+MGM7jmVw/GB33oeoCqow70HfXBPDHyvaPGXORjKzc3nj5wMIsXGZjKlxnEwWyUAnEUkA9uFLCDeULCQiXYBo4NsSdaNFJFZV04BhwAoHY3WFqnJy5UrSX55G1ldfIZEReK8ZyScDgvjg2GJO7J1H20Ztuee8exjdYTSxkbFuh1x5W+fBtgVw6R+gQRwACzce4sPv9nH3xZ3o1rIaBuY3xpw1x5KFquaLyERgPr5HZ19V1Q0iMgVYoaqz/UXHAzNUVYvVLRCR3wCfi68BfiXwslOxVjf1esn64gvf46+rVyPRTdg3bgivdjnAutz5hGeGc2m7SxnbcSz9mvcLuA9ixl2+9xRGOBl8ZeTl+O4qmnWBpNsBOJqdx8MfruPccxpy59COLgdojCmLo+9ZqOpcYG6JbZNLrD9WRt0FQC/HgnOB5uZydM4npL/yCrnbt1NwTjOWjuvGtDY7OOH5Lz0a9uB3nX7HZQmX0TC0iuaHqEm+/btvetSbPgKPrxntiU82kn4il1dv7W9zRhhTg9kb3NWgIOsEme+9x5Hp08k/eJDjbWL44NqmfNo+g4YRytj24xjbaWz1jO7qlqMp8PWfoeuV0MH3HuYXW1KZtTKFiUM70qPV6W+Ut8v9jRtRGmPKYMnCQflHjnDkzTfJePvfeI8dY2/Hxrx5nYc17Y9yfqsLeKbTWIa1HkaoJ9TtUJ332W9Bvb6+CuBYTh4Pvb+Ozs0bcNfF1vxkTE1nyQIY9/cNvoWRVXO83JQUjrz6GhmzZqF5uXzXJZRZYzxkd27MmI638EzHq2jZoGXVfFltsPNr2PAh/OQhiG4LwJOfbCL1eA4v3jTorOazNsa4w5JFFcrZvJmDL/6L7PkL8Ap82QM+PT+Mbn0vYVKnsQxsMZAgqWft8gX58OkD0KQNDLobgMVb05iRvJc7LupA79ZVP31qTdOthT3hZWo/SxaVpKqcWL6cnS/8meDlazkZCgv6C5uGd+SSxOt5O+FymoTX/QtimVa8Aqkb4Pq3ICSCrFP5PPTBOjrERnHP8E5uR2eMCZAliwpSr5eUuR9wYOoLNNx2kKxI+HxoOMHXXMGVfcZxf0y32jHshpNOHIYv/gDth8K5VwDw1NxN7D96kll3XEB4iDU/GVNbWLI4S7k5J1g5/c/ovz8i+lA2J5rAt9cm0GH8L7mv82XuTyRUk3z+OOSeKJrU6L/bDvP2sj38anAC/dpGux2dMeYsWLII0PZ961kz7VlazFlBk+Ne9rQIZs/Eixl4w68Z2rS92+HVPPtWwao34fw7IbYLJ07lc//7a0loFsV9l3ZxO7pq9drI19wOocosu+19t0OoEnXlPKD6zsWSxRlk52Wz8LtZpE5/jT7fHKTrKUjp0hS97UaGXflLQjy1dHwmp3m9MHcSRMXCRQ8A8Md5m9mXeZKZt59vzU/G1EKWLEpQVdakrWHBf98k8r0FXLg6jw5eyEjqTKuJk+iaeKHbIdZ8a96BfStgzL8gvBFLd6Qz/dvd3DaoHf3bNXU7OmNMBViy8MsKU6ZvmM6yL9+h38I9jNqsqCcIvWwoHf93EmHta+FQ4G7IOQoLH4X4/tBrHCdzC3jg/bW0aRrJpBH1q/nJmLqk3ieL1OxUpg/MQfILuPK+Z7hrl1IQGUbj266n+a2/ICQuzu0Qa5ev/uh7Cupn70FQEM/O38ju9Gze+dVAIkMD/+f27u3nOxikMeZs1ftkEXE4i5s/y6dtmkJMNLH33Ub0uHF4GtbBgfyclrYFlk2F826Gln1ZsesIr/13JzcNbMv5HWLcjs4YUwn1Plk0OCee5seDSI8N5oKFXxAUFuZ2SLWTKnx6P4RGwcWTyckr4P5Za2nVJIIHLzvX7eiMMZVU75OFhIaS2ioSwBJFZWz6D+z4Ei57FqKa8ee5m9hx+ARv/zKJqLB6/8/MmFqvng1UZByRmw3zH4G47pD4c1btyWDa1zsYP6ANgzo2czs6Y0wVsF/5TOUt+Ssc3QO3fkKOV5j03hrOaRTOw6Os+cmYusLuLEzlZOyGJX+B7ldDuwv56+ffsz3tBE9d04uG4fbSojF1hSULUznzHwYJgkt/z5q9mbz41XauS4znos6xbkdmjKlCjiYLERkpIltEZJuIPFjK/udFZLX/s1VEMkvsbyQi+0TkH07GaSpo+yLYPAcG38epqHOYNGsNcQ3DeeTybm5HZoypYo71WYiIB3gBuARIAZJFZLaqbiwso6r3Fit/F9C3xGGeAL5yKkZTCQV58OmDEJ0AF9zFPxZtY+uhLF67tT+NI6z5yZi6xsk7iwHANlXdoaq5wAzgqjOUHw+8U7giIv2A5sBnDsZoKmrZi3B4C4x8mvWHcvjnl9u5+rxWDD3X3ng3pi5yMlm0AvYWW0/xb/sREWkLJACL/OtBwHPApDN9gYhMEJEVIrIiLS2tSoI2ATh+CL58GjpdSm6HS5k0ay0xUaE8ekV3tyMzxjjEyWRR2jRxWkbZccAsVS3wr/8vMFdV95ZR3ncw1ZdUNVFVE2NjrUO12ix8DPJzYMRT/PPLbWw6cIw/jO1J40hrfjKmrnLyPYsUoHWx9XhgfxllxwF3Fls/HxgsIv8LNABCRSRLVX/USV4VZtzl+414hBMHr2v2Loc1/4ZB97ApL45/LPqGq/q05JJuzd2OzBjjICeTRTLQSUQSgH34EsINJQuJSBcgGvi2cJuq/qzY/luBRKcShTkL3gLfpEYNW5B34X1MmraGJpEhPHalNT8ZU9c51gylqvnARGA+sAmYqaobRGSKiIwuVnQ8MENVy2qiMjXFd2/CgdVwyRO8+O0h1u87xu/H9CA6KtTtyIwxDnN0uA9VnQvMLbFtcon1x8o5xuvA61UcmjlbJzPg8ynQ5gK2xo3gb+8u4fJeLRjZo4XbkRljqoG9wW0C88VTcDKD/BFPM2nWWhqEBzNltDU/GVNfWLIw5Tu0AZKnQeLPefn7BqxJOcqUq7oT08CGdDemvggoWYhID6cDMTWUKsy9H8Ibs6PnPTy/cCsju5/D5T2t+cmY+iTQO4upIrJcRP5XRJo4GpGpWTZ8ALu/wTvsd9w3Zw+RoR6eGNMDkdJeozHG1FUBJQtVvRD4Gb73JlaIyL9F5BJHIzPuyz0Bn/0OzunFq9mD+W5PJo+P7k5sQ2t+Mqa+CbjPQlW/B34LPABcBPxNRDaLyNVOBWdc9vVzcGwf+y6YwrMLtjG8a3NG927pdlTGGBcE2mfRS0Sex/e+xDDgSlXt6l9+3sH4jFvSt8N//472vI57/htGWHAQT4615idj6qtA7yz+AawCeqvqnaq6CkBV9+O72zB1zfyHwRPKu9G/InlXBo9e2Z24RuFuR2WMcUm5L+X556XYq6pvlra/rO2m+r028rWqOdDWz2DrPDIG/ZbHvkhnaJdYrj6v1AGDjTH1RLl3Fv6RYGNExMZ0qA/yT8G8B9GYTty5PYmQoCCevLqnNT8ZU88FOtzHbmCJiMwGThRuVNU/OxKVqZDrX/SNxfju7edX/CBL/wlHtvN5v3/y3yXH+eM1vWjROKKKIjTG1FaBJov9/k8Q0NC5cIyrju2Hr54lu/0I/i85hiGdm3JtYrzbURljaoCAkoWqPu50IKYGWDAZ9ebzcPZ4gkR4ypqfjDF+ASULEYkF7ge6A0WPxKjqMIfiMtVt939h3Xts6DCBjzaE8oex59KqiTU/GWN8An109m1gM755sh8HduGb3MjUBd4CmHs/+Q1bceu2wVzQIYYbBrRxOypjTA0SaLKIUdVXgDxV/UpVfw4MdDAuU51WvAqH1vGv0NvI1lCeuaaXNT8ZY04TaAd3nv/nARG5HF9nd53p+ayy9xNqoxPpsOj3HIoZwHP7ujLlqnNp3TTS7aiMMTVMoMni9yLSGLgP+DvQCLjXsahM9Vn0BHrqOLefvJ6khBhuTGrrdkTGmBoo0Keh5vgXjwJDAz24iIwE/gp4gGmq+nSJ/c8XO14kEKeqTUSkD/AvfEmpAPiDqr4b6PeaAB1Yg658nQUNx7AlM555P+1FUJA1PxljfizQp6FeA7Tkdn/fRVl1PMALwCVACpAsIrNVdWOx+vcWK38X0Ne/mg3crKrfi0hLYKWIzFfVzEDiNQHwT2p0KjSa36SNYtIVXWgbE+V2VMaYGirQZqg5xZbDgbH4+i3OZACwTVV3AIjIDOAqYGMZ5ccDjwKo6tbCjaq6X0RSgVjAkkVVWTsT9i7lSe6gS7t4br2gndsRGWNqsECbod4vvi4i7wALy6nWCthbbD0FSCqtoIi0xfdY7qJS9g0AQoHtgcRqAnDqOLpgMrtCuzAzewif/rS3NT8ZY84o4MmPSugElPcgfmlXnx81ZfmNA2b5By384QAiLYA3gdtU1fujLxCZICIrRGRFWlpaAGEbAL76I5J1kHuP/4z7Lu1KQjNrfjLGnFmgkx8dF5FjhR/gP/hmzDuTFHzTsBaKp+ymq3HAOyW+sxHwCfBbVV1aWiVVfUlVE1U1MTY2NpBTMYe/R5f+i48YirRO5OcXJrgdkTGmFgi0GaoigwcmA51EJAHYhy8h3FCykIh0AaKBb4ttCwU+BN5Q1fcq8N2mNKropw+QQyjP5F/Pmz/tjcean4wxAQj0zmKs/z2LwvUmIjLmTHVUNR+YCMzHNx3rTFXdICJTRGR0saLjgRmqWryJ6jpgCHCriKz2f/oEeE6mLFvmIts/59lTV3Pz8AF0jGvgdkTGmFoi0KehHlXVDwtXVDVTRB4FPjpTJVWdC8wtsW1yifXHSqn3FvBWgLGZQOSdpODTB9lFPKvP+SkzB1vzkzEmcIF2cJdWLtBEY2qC//4Dz9E9PJ5/K09d249gT0WfbTAmxqlqAAAaoElEQVTG1EeBXjFWiMifRaSDiLT3v3m90snATBXK3EvB4j/xScEABgwbQ5dzbP4qY8zZCTRZ3AXkAu8CM4GTwJ1OBWWqVu6nD5NX4OW9pndw+0Ud3A7HGFMLBfo01AngQYdjMU7Y8RWhW2bzfP61PDDuEkKs+ckYUwGBPg21QESaFFuPFpH5zoVlqkRBHlkf3cdebyzBg/+Pri0auR2RMaaWCrSTulnxQfxUNUNE4hyKyVTQ5PRJ/qVvAMheMpUGx77nuQYP89DFPdwLzBhT6wXaJuEVkaLhPUSkHWUP3WFqgqw05MunWeztxTU33E5osDU/GWMqLtA7i0eAb0TkK//6EGCCMyGZqrD//QeJLTjJ9vN+y5D4JuVXMMaYMwi0g3ueiCTiSxCrgY/xPRFlaqCs7ctouXMW74WN5YYrhrsdjjGmDgh08qNfAnfjGwxwNTAQ31hOw5wLzVSIKkdm3U22NqHruCcIC/a4HZExpg4ItCH7bqA/sFtVh+Kb0c7GBK+BMvODaXNyE8s73k2P9q3Lr2CMMQEINFnkqGoOgIiEqepmoItzYZmKOOX1cK53O+s95zL8+v9zOxxjTB0SaAd3iv89i4+ABSKSQfnTqppqdiLXSxPJIvXyPxEeakN3GWOqTqAd3GP9i4+JyBdAY2CeY1GZs7b+648ZHLSWL/Q8hp432O1wjDF1zFn/+qmqX5VfylSnE8cyiFl0H3s0jiYhBeVXMMaYs2RvatUB69/4Nc29h9kWlEBo0I+mKjfGmEqzZFHLbVjyCUmHP2B58+toHpLtdjjGmDrKkkUtlp11lCYL7yVFzqH3Lc+5HY4xpg5zNFmIyEgR2SIi20TkR0Oci8jzxebY3ioimcX23SIi3/s/tzgZZ2217o37aKWHOHrpX4iIsgmNjDHOcez5ShHxAC8AlwApQLKIzFbVjYVlVPXeYuXvwveyHyLSFHgUSMQ3YOFKf90Mp+KtbTYtm0//Q7NYFnsNSedf5nY4xpg6zsmH8QcA21R1B4CIzACuAjaWUX48vgQBMAJYoKpH/HUXACOBdxyMt9Y4eeI4Debdw8GgOHrc8uei7VNingV80xkaY0xVcrIZqhWwt9h6in/bj4hIWyABWHS2deujNW9MorXu58jFfyKqoY0oa4xxnpPJQkrZVtYcGOOAWapa+JJAQHVFZIKIrBCRFWlp9WOoqs3JCxlwcAbLYsbQ48LRbodjjKknnEwWKUDxkeziKXuIkHGc3sQUUF1VfUlVE1U1MTY2tpLh1nw5J08Q8endpEozut/yF7fDMcbUI04mi2Sgk4gkiEgovoQwu2QhEekCROMb8rzQfOBS/1zf0cCl/m312ndvPEhbbwppQ/9Ig0bRbodjjKlHHOvgVtV8EZmI7yLvAV5V1Q0iMgVYoaqFiWM8MENVtVjdIyLyBL6EAzClsLO7vtq66ksG7H+T5U2vYMBFV7sdjjGmnnF0aFJVnQvMLbFtcon1x8qo+yrwqmPB1SKncrIJnTORw9KUc2/5m9vhGGPqIXuDuxZY9ebDtPPu5eBFz9CoSYzb4Rhj6iFLFjXcttVf0z9lOssbX0bvode6HY4xpp6yZFGD5Z7KwTN7IhnSmC63/N3tcIwx9Zglixps5Vu/JcG7i30XPkXjpnX/0WBjTM1lyaKG2r5uKYl7XiW50SX0GT7e7XCMMfWcJYsaKC/3FHz0PxyVhnS6+QW3wzHGGEsWNdGKtyfToWAHe87/PU2aNXc7HGOMcfY9C3P2dm5Mpt+ul1nZaCj9RtzkdjjG1Ap5eXmkpKSQk5Pjdig1Vnh4OPHx8YSEhFSoviWLGiQ/L5f8D+4gS6JIuMman4wJVEpKCg0bNqRdu3aIlDYOaf2mqqSnp5OSkkJCQkKFjmHNUDVI8r8fp1P+NnYOmELTOBuR3ZhA5eTkEBMTY4miDCJCTExMpe68LFnUELs3r6TfjqmsihpCv1G3uR2OMbXO2SaK61/8lutf/Lb8gnVEZROpJYsaoCA/n5xZ/8MJiaDNTf90OxxjTAU0aNCgaHnkyJE0adKEK664otSyd955J3369KFbt25ERETQp08f+vTpw6xZs87qO1etWsW8efMqFXegrM+iBkie8QQD87ewIvFZEs9pXX4FY0yNNmnSJLKzs3nxxRdL3f/CC74+yV27dnHFFVewevXqCn3PqlWrWL9+PSNHjqxwrIGyOwuX7f1+DX2+f4HvIi+g36hfuh2OMaYKXHzxxTRs2LBCdb///ntGjBhBv379GDJkCFu3bgVgxowZ9OjRg969ezN06FBOnjzJlClTePvttyt0V3K27M7CRQX5+WTNvINGEkrrG6ciQZa7jamsx/+zgY37j5VbbuMBX5lA+i26tWzEo1d2r3RsgZgwYQLTpk2jQ4cOLFmyhIkTJ/LZZ5/x+OOP8+WXX9K8eXMyMzOJiIhg8uTJrF+/nr/8xfmZMy1ZuCh55lMMzNtIct+n6N+ybaWP9+7t51dBVMYYt2RmZrJ06VKuueaaom35+fkADBo0iJtvvplrr72Wq6+u/gnQLFm4JGXbenpv+RtrIpNIvPIOt8Mxps4I9A6g8I6iJv2Spao0a9as1D6Ml19+mWXLljFnzhx69+7N2rVrqzU2a/dwgbeggGMz7yBPgmlx44vW/GSMASA6OpoWLVrw4YcfAuD1elmzZg0AO3bsYODAgTzxxBNER0ezb98+GjZsyPHjx6slNrtKuSD5vT/SLXcdm3s/RFyrir1NaYypuQYPHsy1117L559/Tnx8PPPnzw+47owZM5g6dSq9e/eme/fuzJkzB4B7772Xnj170rNnT4YPH06PHj0YNmwYa9asoW/fvrW7g1tERgJ/BTzANFV9upQy1wGPAQqsUdUb/Nv/CFyOL6EtAO5WVXUy3uqwf+dmem56nrURifS/aqLb4RhjqkhWVlbR8tdffx1QnXbt2rF+/frTtrVv377U5DJ79uwfbYuNjWXFihVnGWnFOJYsRMQDvABcAqQAySIyW1U3FivTCXgIGKSqGSIS599+ATAI6OUv+g1wEfClU/FWB/UWkPHOBBoRRNzPrPnJGDfVpL6K2sDJq9UAYJuq7lDVXGAGcFWJMr8CXlDVDABVTfVvVyAcCAXCgBDgkIOxVovk9/9M99w1bOh5P+e07uh2OMYYEzAnk0UrYG+x9RT/tuI6A51FZImILPU3W6Gq3wJfAAf8n/mquqnkF4jIBBFZISIr0tLSHDmJqnJg9xa6r/8T68L6MuDqe9wOxxhjzoqTyaK0UatK9jkEA52AnwDjgWki0kREOgJdgXh8CWaYiAz50cFUX1LVRFVNjI2tuXNUq9fL4X/fgaDEjLfmJ2NM7ePkVSsFKD7QUTywv5QyH6tqnqruBLbgSx5jgaWqmqWqWcCnwEAHY3VU8od/o+epVazr9htatuvidjjGGHPWnEwWyUAnEUkQkVBgHFCyO/8jYCiAiDTD1yy1A9gDXCQiwSISgq9z+0fNULXBoZTtdF37NBtCe9H/p/e5HY4xptBrl/s+JiCOJQtVzQcmAvPxXehnquoGEZkiIqP9xeYD6SKyEV8fxSRVTQdmAduBdcAafI/U/sepWJ2iXi8H37odD16ajHuJII/H7ZCMMQ6p7iHKP/zwQ5599tlKxx0oR9+zUNW5wNwS2yYXW1bg1/5P8TIFwO1OxlYdVsz+J/1zkll67v0MbN/V7XCMMdWkqoYoz8/PJzi49Mv02LFjqybYAFlPq0PS9u+iy+on2RjSgwHXPeh2OMaYalSZIcovvPBCHnnkEYYMGcI//vEPPv74Y5KSkujbty+XXnopqam+NwymTZvGPff4nqy88cYbufvuu7ngggto37590XAhVckGEnSAer3se+sOztVcGl4/1ZqfjKlOnz4IB9eVX+6gfyC+QPotzukJl/1oAArHHDt2jMWLFwOQkZHB6NGjERGmTp3Kc889xzPPPPOjOqmpqSxZsoR169Zx3XXXVfmdhyULB6yc8xKJ2d+ytPOvGdixp9vhGGNqmXHjxhUt79mzh+uuu46DBw9y6tQpOnfuXGqdMWPGICL06tWLffv2VXlMliyq2OGDe+i0agqbg7vS//pH3A7HmPon0DuAwjuK2z5xLpYKioqKKlq+8847efjhhxk1ahQLFy7k6adLP7+wsLCiZSeG0bM+iyqkXi973/wfwjWXiGun4imjY8oYYwJ19OhRWrVqhaoyffp01+KwZFGFVn36Kn1PfMN3Hf6Htl36uB2OMcYllRmivKTHHnuMsWPHctFFF9G8efMqjPLsSB0Y9RuAxMREra6hekuTfiiFoH+dT1rwObR/YAnBIaGuxWJMfbNp0ya6dj3Lx9NrcDOUU0r7cxKRlaqaWF5dayepIrvf/F96aDZHr5lqicKY2qAeJYmqYM1QVWDVvNc5L+srViXcTruu/dwOxxhjqpwli0rKSDtA26WT2ebpQL8bHnU7HGOMcYQli0ra/sadNNQsgsb+i5DQsPIrGGNMLWTJohK+++wtEo9/zsq2v6R9jyS3wzHGGMdYsqigo+mHaP3fR9juSSDxxifcDscYc5Zum3cbt827ze0wag1LFhW09Y2JNNbjcNU/rfnJGFM0RPnq1as5//zz6d69O7169eLdd9/9UdmqGKIcYNWqVcybN69K4i+PPTpbAWsWzaD/0c9Y2voXDOx1gdvhGGNqkMjISN544w06derE/v376devHyNGjKBJkyZFZQIdorw8q1atYv369YwcObJKYj8Tu7M4S0czDtNi8UPsDGrLeTc96XY4xpgapnPnznTq1AmAli1bEhcXR1paWsD1v//+e0aMGEG/fv0YMmQIW7duBWDGjBn06NGD3r17M3ToUE6ePMmUKVN4++23K3RXcrbszuIsbZl+F+dpJkevnE5oWLjb4RhjSnhm+TNsPrK53HKFZQLptzi36bk8MOCBs45l+fLl5Obm0qFDh4DrTJgwgWnTptGhQweWLFnCxIkT+eyzz3j88cf58ssvad68OZmZmURERDB58mTWr1/PX/7yl7OO7WxZsjgLa7+YxYDMuXzb6hbO7zvE7XCMMTXYgQMHuOmmm5g+fTpBQYE14mRmZrJ06VKuueaaom35+fkADBo0iJtvvplrr72Wq6++2pGYz8TRZCEiI4G/Ah5gmqr+aGxdEbkOeAxQfHNt3+Df3gaYBrT27xulqrucjPdMjh89QvOv7md3UGv63vSUW2EYY8oR6B1A4R3FayNfq/IYjh07xuWXX87vf/97Bg4cGHA9VaVZs2al9mG8/PLLLFu2jDlz5tC7d2/Wrl1blSGXy7E+CxHxAC8AlwHdgPEi0q1EmU7AQ8AgVe0O3FNs9xvAs6raFRgApDoVayA2Tr+bZnqEnFF/IzwiqvwKxph6KTc3l7FjxxbdBZyN6OhoWrRoUTQtqtfrZc2aNQDs2LGDgQMH8sQTTxAdHc2+ffto2LAhx48fr/JzKI2THdwDgG2qukNVc4EZwFUlyvwKeEFVMwBUNRXAn1SCVXWBf3uWqmY7GOsZrVv8MUlHZpPc4ga6JA5zKwxjTC0wc+ZMFi9ezOuvv170SOzZPO00Y8YMpk6dSu/evenevTtz5swB4N5776Vnz5707NmT4cOH06NHD4YNG8aaNWvo27dvre7gbgXsLbaeApR8zbkzgIgswddU9ZiqzvNvzxSRD4AEYCHwoKoWOBhvqbKOZdBs0X3slZb0ufmP1f31xphaIisrC4Abb7yRG2+8MaA67dq1Y/369adta9++fanzX8yePftH22JjY6muqRmcTBZSyraSk2cEA52AnwDxwNci0sO/fTDQF9gDvAvcCrxy2heITAAmALRp06bqIi9mwxu/pr8eZuuo92gd2cCR7zDGVD8n+irqMieboVLwdU4Xigf2l1LmY1XNU9WdwBZ8ySMF+M7fhJUPfAScV/ILVPUlVU1U1cTY2NgqP4H1S/5D0uEPWH7O9Zw74JIqP74xxtQWTiaLZKCTiCSISCgwDih5H/URMBRARJrha37a4a8bLSKFGWAYsNHBWH8kO+so0QvvI0Va0PvmP1XnVxtjTI3jWLLw3xFMBOYDm4CZqrpBRKaIyGh/sflAuohsBL4AJqlqur9v4jfA5yKyDl+T1stOxVqatdPvo5Ue4tilzxMR1bA6v9oYY2ocR9+zUNW5wNwS2yYXW1bg1/5PyboLgF5OxleWjUvnMSB1FsvifkrS+Ze5EYIxxtQoNjZUCSdPHKfR/Hs4GBRHz1v+7HY4xhiH7L7pZnbfdLPbYdQalixKWPPGJOL1AEeGP0dkg8Zuh2OMqSWqe4jyDz/8kGeffbbK4i+PjQ1VzOblCxhwcAbLmo0hadCVbodjjKmFqnKI8vz8fIKDS79Mjx07tuqDPwNLFn452VlEzrubVGlG91ucH8HRGFM3de7cuWi5+BDlxZPFmVx44YVcdNFFfP3111x99dUkJCTw5JNPkpubS2xsLG+99RZxcXFMmzataMTZG2+8kZiYGJKTkzl48CDPPfdclScTSxbA9S9+yxWpU7nJu491w17nnEbRbodkjKmgg08+yalN5Q9RnrPZVyaQfouwrudyzsMPn3UsFRmiHHwDES5evBiAjIwMRo8ejYgwdepUnnvuOZ555pkf1UlNTWXJkiWsW7eO6667zpKFE1qe2MAN+R+zvOkVDBhSvbd2xpi6qSJDlBcaN25c0fKePXu47rrrOHjwIKdOnTrtzqW4MWPGICL06tWLffv2VSr20tT7ZHEqJ5v/y3yWNGnCubf8ze1wjDGVFOgdQOEdRds336jyGCo6RHmhqKgfRra+8847efjhhxk1ahQLFy7k6ad/NNMDAGFhYUXLvrcSqla9fxoq/aBvrMN1QV1o1CTG5WiMMbVdZYYoL83Ro0dp1aoVqsr06dOrIMKKqfd3Fi3bdWFci2loUDA2+pMxprIKhyhPT0/n9ddfBygarrwiHnvsMcaOHUt8fDwDBgzgwIEDVRht4MSJ2xU3JCYmakWH6r3+xW8BePf286syJGNMNdm0aRNdu3Y9qzpONkPVVKX9OYnISlVNLK9uvb+zAEsSxtRH9SlJVIV632dhjDGmfJYsjDHGlMuShTGmTqgr/a9OqeyfjyULY0ytFx4eTnp6uiWMMqgq6enphIeHV/gY1sFtjKn14uPjSUlJIS0tze1Qaqzw8HDi4+MrXN+ShTGm1gsJCSEhIcHtMOo0a4YyxhhTLksWxhhjymXJwhhjTLnqzHAfIpIG7K7EIZoBh6soHDfVlfMAO5eaqq6cS105D6jcubRV1djyCtWZZFFZIrIikPFRarq6ch5g51JT1ZVzqSvnAdVzLtYMZYwxplyWLIwxxpTLksUPXnI7gCpSV84D7FxqqrpyLnXlPKAazsX6LIwxxpTL7iyMMcaUy5KFn4g8ISJrRWS1iHwmIi3djqmiRORZEdnsP58PRaSJ2zFVlIhcKyIbRMQrIrXuyRURGSkiW0Rkm4g86HY8lSEir4pIqoisdzuWyhCR1iLyhYhs8v/butvtmCpKRMJFZLmIrPGfy+OOfZc1Q/mISCNVPeZf/j+gm6re4XJYFSIilwKLVDVfRJ4BUNUHXA6rQkSkK+AFXgR+o6oVmzvXBSLiAbYClwApQDIwXlU3uhpYBYnIECALeENVe7gdT0WJSAughaquEpGGwEpgTG38exERAaJUNUtEQoBvgLtVdWlVf5fdWfgVJgq/KKDWZlFV/UxV8/2rS4GKDzXpMlXdpKpb3I6jggYA21R1h6rmAjOAq1yOqcJUdTFwxO04KktVD6jqKv/ycWAT0MrdqCpGfbL8qyH+jyPXLksWxYjIH0RkL/AzYLLb8VSRnwOfuh1EPdUK2FtsPYVaelGqq0SkHdAXWOZuJBUnIh4RWQ2kAgtU1ZFzqVfJQkQWisj6Uj5XAajqI6raGngbmOhutGdW3rn4yzwC5OM7nxorkHOppaSUbbX2jrWuEZEGwPvAPSVaFmoVVS1Q1T74WhAGiIgjTYT1aj4LVR0eYNF/A58AjzoYTqWUdy4icgtwBXCx1vCOqbP4e6ltUoDWxdbjgf0uxWKK8bfvvw+8raofuB1PVVDVTBH5EhgJVPlDCPXqzuJMRKRTsdXRwGa3YqksERkJPACMVtVst+Opx5KBTiKSICKhwDhgtssx1Xv+TuFXgE2q+me346kMEYktfNpRRCKA4Th07bKnofxE5H2gC74nb3YDd6jqPnejqhgR2QaEAen+TUtr8ZNdY4G/A7FAJrBaVUe4G1XgRGQU8BfAA7yqqn9wOaQKE5F3gJ/gG+H0EPCoqr7ialAVICIXAl8D6/D9fwd4WFXnuhdVxYhIL2A6vn9fQcBMVZ3iyHdZsjDGGFMea4YyxhhTLksWxhhjymXJwhhjTLksWRhjjCmXJQtjjDHlsmRhzFkQkazyS52x/iwRae9fbiAiL4rIdv+IoYtFJElEQv3L9eqlWVOzWbIwppqISHfAo6o7/Jum4RuYr5OqdgduBZr5Bx38HLjelUCNKYUlC2MqQHye9Y9htU5ErvdvDxKRf/rvFOaIyFwR+am/2s+Aj/3lOgBJwG9V1QvgH532E3/Zj/zljakR7DbXmIq5GugD9Mb3RnOyiCwGBgHtgJ5AHL7hr1/11xkEvONf7o7vbfSCMo6/HujvSOTGVIDdWRhTMRcC7/hH/DwEfIXv4n4h8J6qelX1IPBFsTotgLRADu5PIrn+yXmMcZ0lC2MqprThx8+0HeAkEO5f3gD0FpEz/R8MA3IqEJsxVc6ShTEVsxi43j/xTCwwBFiOb1rLa/x9F83xDbxXaBPQEUBVtwMrgMf9o6AiIp0K5/AQkRggTVXzquuEjDkTSxbGVMyHwFpgDbAIuN/f7PQ+vnks1uObN3wZcNRf5xNOTx6/BM4BtonIOuBlfpjvYihQ60ZBNXWXjTprTBUTkQaqmuW/O1gODFLVg/75Br7wr5fVsV14jA+Ah2rx/OOmjrGnoYypenP8E9KEAk/47zhQ1ZMi8ii+ebj3lFXZP1HSR5YoTE1idxbGGGPKZX0WxhhjymXJwhhjTLksWRhjjCmXJQtjjDHlsmRhjDGmXJYsjDHGlOv/AWAe3Bmq1Q33AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores =  np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(np.log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuary' )\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l2\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_['penalty'])\n",
    "print(grid.best_params_['C'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
